////////////////////////////////////////////////////////////////////////////
//
//  CryEngine Source File.
//  Copyright (C), Crytek.
// -------------------------------------------------------------------------
//  File name:   StreamAsyncFileRequest.cpp
//  Created:     22/07/2010 by Timur.
//  Description: 
// -------------------------------------------------------------------------
//  History:
//
////////////////////////////////////////////////////////////////////////////

#include <StdAfx.h>
#include "StreamAsyncFileRequest.h"
#include "StreamEngine.h"
#include "IDiskProfiler.h"
#include "../System.h"
#include "../CryPak.h"





extern CMTSafeHeap* g_pPakHeap;
extern IStreamEngine::SStatistics* g_pStreamingStatistcs;

uint64 CAsyncIOFileRequest::s_nSequenceCounter = 0;

#define INPLACE_UNZIP_SAFETY_OFFSET 256

//////////////////////////////////////////////////////////////////////////
CAsyncIOFileRequest::CAsyncIOFileRequest( EStreamTaskType eType )
: m_status(eStatusNotReady),
	m_eType(eType),
	m_nErrorCode(0),
	m_bCanceled(0),
	m_pExternalMemoryBuffer(0),
	m_pOutputMemoryBuffer(0),
	m_bCompressedBuffer(false),
	m_bDecompressInPlace(false),
	m_bNoSorting(false),
	m_bStatsUpdated(false),
	m_nFileSize(0),
	m_nFileSizeCompressed(0),
	m_nRequestedOffset(0),
	m_nRequestedSize(0),
	m_nSizeOnMedia(0),
	m_nSortKey(0),
	m_nDiskOffset(0),
	m_nReadHeadOffsetKB(0),
	m_completionTime(0),
	m_pCallback(0),
	m_maxFileSize(0),
	m_eMediaType(eStreamSourceTypeHDD),
	m_bSortKeyComputed(false)
{
	m_startTime = gEnv->pTimer->GetAsyncTime();

	if (g_pStreamingStatistcs)
	{
		IStreamEngine::SStatistics &stats = *g_pStreamingStatistcs;
		CryInterlockedIncrement( &stats.nOpenRequestCount );
		CryInterlockedIncrement( &stats.typeInfo[m_eType].nOpenRequestCount );
	}
}

//////////////////////////////////////////////////////////////////////////
CAsyncIOFileRequest::~CAsyncIOFileRequest()
{
	FreeBuffer();

	Done();
}

//////////////////////////////////////////////////////////////////////////
void CAsyncIOFileRequest::Cancel()
{
	if (!m_bCanceled)
	{
		// Stall until the file request isn't using the external buffer anymore
		CryOptionalAutoLock<CryCriticalSection> readLock(m_externalBufferLock, m_pExternalMemoryBuffer != NULL);

		Done();
		m_bCanceled = true;
	}
}

//////////////////////////////////////////////////////////////////////////
void CAsyncIOFileRequest::Done()
{
	if (!m_bCanceled && m_status != eStatusDone)
	{
		m_status = eStatusDone;
		
		if (g_pStreamingStatistcs)
		{
			IStreamEngine::SStatistics &stats = *g_pStreamingStatistcs;
			CryInterlockedDecrement( &stats.nOpenRequestCount );
			CryInterlockedDecrement( &stats.typeInfo[m_eType].nOpenRequestCount );
		}
	}
}

//////////////////////////////////////////////////////////////////////////
void CAsyncIOFileRequest::FreeBuffer()
{
	if (m_pMemoryBuffer && m_pMemoryBuffer->GetData())
	{
		CStreamEngine* pStreamEngine = GetStreamEngine();












		pStreamEngine->ReportTempMemAlloc( 0,m_pMemoryBuffer->GetSize() );
		m_pMemoryBuffer = 0;






















	}

	// Update Streaming statistics.
	if (g_pStreamingStatistcs && m_nSizeOnMedia != 0 && m_bStatsUpdated)
	{
		m_bStatsUpdated = false;
		int nSize = (int)m_nSizeOnMedia;
		IStreamEngine::SStatistics &stats = *g_pStreamingStatistcs;
		CryInterlockedAdd( &stats.nPendingReadBytes,-nSize );
		CryInterlockedAdd( &stats.typeInfo[m_eType].nPendingReadBytes,-nSize );
	}
}

//////////////////////////////////////////////////////////////////////////
void CAsyncIOFileRequest::ReadFile()
{
	// The lock only needs to be acquired if an external memory buffer whose lifetime we don't manage
	// is being used.
	CryOptionalAutoLock<CryCriticalSection> readLock(m_externalBufferLock, m_pExternalMemoryBuffer != NULL);

	if (m_bCanceled)
		return;

	CTimeValue t0 = gEnv->pTimer->GetAsyncTime();

	CryStackStringT<char, MAX_PATH> fileName(m_strFileName.c_str());
	ICryPak* pIPak = gEnv ? gEnv->pCryPak : NULL;
	if (m_pReadStream && m_pReadStream->GetParams().nFlags & IStreamEngine::FLAGS_FILE_ON_DISK)
	{
		pIPak = 0;

		const int g_nMaxPath = 0x800;
		char szFullPathBuf[g_nMaxPath];
		fileName = gEnv->pCryPak->AdjustFileName(m_strFileName.c_str(), szFullPathBuf, ICryPak::FOPEN_HINT_QUIET);
	}

	CCryFile file(pIPak);
	if (!file.Open( fileName.c_str(),"rb",ICryPak::FOPEN_FORSTREAMING ))
	{
		ReadFailed(ERROR_CANT_OPEN_FILE); // can't open file
		return;
	}

	m_nFileSize = file.GetLength();

	if (m_maxFileSize != 0 && m_nFileSize > m_maxFileSize)
	{
		Cancel();
		return;
	}

	if (m_nRequestedOffset >= m_nFileSize)
	{
		// offset out of range
		ReadFailed(ERROR_OFFSET_OUT_OF_RANGE);
		return;
	}

	if (m_nRequestedOffset + m_nRequestedSize > m_nFileSize)
	{
		ReadFailed(ERROR_SIZE_OUT_OF_RANGE);
		return;
	}

	if (m_nRequestedSize == 0)
	{
		// by default, we read the whole file
		m_nRequestedSize = m_nFileSize - m_nRequestedOffset;
	}

	CCachedFileDataPtr pZipEntry = ((CCryPak*)(gEnv->pCryPak))->GetOpenedFileDataInZip( file.GetHandle() );

	if (!pZipEntry || pZipEntry->m_pFileEntry->nMethod == 0)
	{
		m_bCompressedBuffer = false;
		m_nFileSizeCompressed = m_nFileSize;

		if (!m_pExternalMemoryBuffer)
		{
#if defined(INCLUDE_MEMSTAT_CONTEXTS)
			char usageHint[512];
			sprintf_s(usageHint, 512, "AsyncIO TempBuffer: %s", m_strFileName.c_str());
#else
			const char* usageHint = "AsyncIO TempBuffer";
#endif
			// for decompression in callback, we don't care about auto allocation of buffer. user should take care of it
			m_pMemoryBuffer = gEnv->pCryPak->PoolAllocMemoryBlock( m_nRequestedSize,usageHint );
			if (!m_pMemoryBuffer->GetData())
			{
				ReadFailed(ERROR_OUT_OF_MEMORY);
				return;
			}
			GetStreamEngine()->ReportTempMemAlloc( m_nRequestedSize,0 );

			m_pOutputMemoryBuffer = m_pMemoryBuffer->GetData();
		}
		else
		{
			m_pOutputMemoryBuffer = m_pExternalMemoryBuffer;
		}
		// If this is normal file on disk or uncompressed zip file, simple read all data directly.
		file.Seek( m_nRequestedOffset,SEEK_SET );
		if (file.ReadRaw( m_pOutputMemoryBuffer,m_nRequestedSize ) != m_nRequestedSize)
		{
			ReadFailed(ERROR_REFSTREAM_ERROR);
			return;
		}
		m_nSizeOnMedia = m_nRequestedSize;
	}
	else
	{
		//pZipEntry->m_pZip->IsInMemory();

		m_bCompressedBuffer = true;
		if (m_nRequestedOffset == 0 && m_nRequestedSize == m_nFileSize)
		{
			m_bDecompressInPlace = true;
		}

		m_nFileSize = pZipEntry->GetFileEntry()->desc.lSizeUncompressed;
		m_nFileSizeCompressed = pZipEntry->GetFileEntry()->desc.lSizeCompressed;

		uint32 nTempDataSize = m_nFileSizeCompressed;
		if (m_bDecompressInPlace)
		{
			// Try in place decompression.
			nTempDataSize = m_nFileSize+INPLACE_UNZIP_SAFETY_OFFSET;
		}

#if defined(INCLUDE_MEMSTAT_CONTEXTS)
		char usageHint[512];
		sprintf_s(usageHint, 512, "AsyncIO Compressed TempBuffer: %s", m_strFileName.c_str());
#else
		const char* usageHint = "AsyncIO Compressed TempBuffer";
#endif

		// This is compressed entry in zip.
		m_pMemoryBuffer = gEnv->pCryPak->PoolAllocMemoryBlock( nTempDataSize,usageHint );

		void *pCompressedDataPtr = m_pMemoryBuffer->GetData();

		if (!pCompressedDataPtr)
		{
			ReadFailed(ERROR_OUT_OF_MEMORY);
			return;
		}
		
		GetStreamEngine()->ReportTempMemAlloc( nTempDataSize,0 );

		if (m_bDecompressInPlace)
		{
			pCompressedDataPtr = (char*)pCompressedDataPtr + (m_nFileSize-m_nFileSizeCompressed) + INPLACE_UNZIP_SAFETY_OFFSET;
		}

		// Read compressed data from file into the temporary buffer.
		if (!pZipEntry->GetDataTo( pCompressedDataPtr,m_nFileSizeCompressed,false ))
		{
			ReadFailed(ERROR_ZIP_CACHE_FAILURE);
			return;
		}
		m_nSizeOnMedia = m_nFileSizeCompressed;
	}
	file.Close();

	CTimeValue t1 = gEnv->pTimer->GetAsyncTime();
	m_readTime = t1-t0;

	if (!m_bCompressedBuffer)
	{
		m_completionTime = (t1 - m_startTime).GetMilliSeconds();
	}

	ReadComplete();
}

//////////////////////////////////////////////////////////////////////////
bool CAsyncIOFileRequest::Decompress()
{
	FUNCTION_PROFILER( gEnv->pSystem, PROFILE_SYSTEM );

	CryOptionalAutoLock<CryCriticalSection> readLock(m_externalBufferLock, m_pExternalMemoryBuffer != NULL);

	if (m_bCanceled || m_nErrorCode != 0)
		return true;

	if (!m_bCompressedBuffer)
		return true;

	PROFILE_DISK_TASK_TYPE( (m_pReadStream)?m_pReadStream->GetCallerType():eStreamTaskTypeTexture );

	CTimeValue t0 = gEnv->pTimer->GetAsyncTime();

	if (m_bCompressedBuffer)
	{
		if (!m_bDecompressInPlace)
		{
			void *pCompressedDataPtr = m_pMemoryBuffer->GetData();
			IMemoryBlock_AutoPtr pMemoryBufferUncompressed = gEnv->pCryPak->PoolAllocMemoryBlock( m_nFileSize,"StreamTask:UncompressedBuffer" );
			unsigned long nUncompressed = m_nFileSize;
			if (0 != ZipDir::ZipRawUncompress( g_pPakHeap,pMemoryBufferUncompressed->GetData(),&nUncompressed,pCompressedDataPtr,m_nFileSizeCompressed ) ||
				(nUncompressed != m_nFileSize))
			{
				// Decompression failed.
				ReadFailed( ERROR_ZIP_CACHE_FAILURE );
				return false;
			}

			GetStreamEngine()->ReportTempMemAlloc( m_nFileSize,m_nFileSizeCompressed );

			// Replace our stored memory buffer with uncompressed one.
			// Release compressed memory buffer here.
			m_pMemoryBuffer = pMemoryBufferUncompressed;

			assert( m_nRequestedOffset+m_nRequestedSize <= m_nFileSize );

			if (!m_pExternalMemoryBuffer)
			{
				// Buffer pointer is within our uncompressed memory block
				m_pOutputMemoryBuffer = (uint8*)pMemoryBufferUncompressed->GetData() + m_nRequestedOffset;
			}
			else
			{
				// Buffer supplied. copy data from uncompressed memory.
				m_pOutputMemoryBuffer = m_pExternalMemoryBuffer;
				memcpy( m_pOutputMemoryBuffer,(uint8*)pMemoryBufferUncompressed->GetData() + m_nRequestedOffset,m_nRequestedSize );
				FreeBuffer();
			}
		}
		else
		{
			// In place decompression.
			void *pCompressedDataPtr = (char*)m_pMemoryBuffer->GetData() + (m_nFileSize-m_nFileSizeCompressed) + INPLACE_UNZIP_SAFETY_OFFSET;
			if (m_pExternalMemoryBuffer)
			{
				m_pOutputMemoryBuffer = m_pExternalMemoryBuffer;
			}
			else
			{
				m_pOutputMemoryBuffer = m_pMemoryBuffer->GetData();
			}
			unsigned long nUncompressed = m_nFileSize;
			if (0 != ZipDir::ZipRawUncompress( g_pPakHeap,m_pOutputMemoryBuffer,&nUncompressed,pCompressedDataPtr,m_nFileSizeCompressed ) ||
					(nUncompressed != m_nFileSize))
			{
				// Decompression failed.
				ReadFailed( ERROR_ZIP_CACHE_FAILURE );
				return false;
			}
			if (m_pExternalMemoryBuffer)
			{
				FreeBuffer();
			}
		}
	}
	CTimeValue t1 = gEnv->pTimer->GetAsyncTime();
	m_unzipTime = t1 - t0;

	m_completionTime = (t1 - m_startTime).GetMilliSeconds();

	m_status = eStatusUnzipComplete;

	return true;
}

//////////////////////////////////////////////////////////////////////////
void CAsyncIOFileRequest::ReadFailed( int nErrorCode )
{
	m_nErrorCode = nErrorCode;
	m_status = eStatusFailed;
}

//////////////////////////////////////////////////////////////////////////
void CAsyncIOFileRequest::ReadComplete()
{
	m_status = eStatusReadComplete;
}

//////////////////////////////////////////////////////////////////////////
CStreamEngine* CAsyncIOFileRequest::GetStreamEngine()
{
	return (CStreamEngine*)GetISystem()->GetStreamEngine();
}

//////////////////////////////////////////////////////////////////////////
EStreamSourceMediaType CAsyncIOFileRequest::GetMediaType()
{
	EStreamSourceMediaType mediaType = eStreamSourceTypeHDD;

	if (m_bSortKeyComputed)
	{
		mediaType = m_eMediaType;		
		return mediaType;
	}
	
	if (m_strFileName.empty())
	{
		mediaType = eStreamSourceTypeMemory;
		return mediaType;
	}

	const int g_nMaxPath = 0x800;
	char szFullPathBuf[g_nMaxPath];
	const char *szFullPath = gEnv->pCryPak->AdjustFileName(m_strFileName.c_str(), szFullPathBuf, ICryPak::FOPEN_HINT_QUIET);

	bool bFileInMemory = false;
	ICryPak* pCryPak = gEnv->pCryPak;
	ZipDir::Cache* pZip = 0;
	unsigned int archFlags;
	ZipDir::FileEntry* pFileEntry = ((CCryPak*)pCryPak)->FindPakFileEntry( szFullPath, archFlags, &pZip, false );
	if (pFileEntry)
	{
		bFileInMemory = pZip->IsInMemory();
	}
	else
	{
		mediaType = eStreamSourceTypeMemory;
		return mediaType;
	}

	// Highest priority for files that are in memory already and can be loaded very fast.
	if (bFileInMemory)
	{
		mediaType = eStreamSourceTypeMemory;		
	}
	else
	{
		// check if data is installed to HDD
		if (pCryPak->IsInstalledToHDD(pZip->GetFilePath()))
		{
			mediaType = eStreamSourceTypeHDD;

		}
		else
		{
			mediaType = eStreamSourceTypeDisc;
		}
	}

	return mediaType;
}

//////////////////////////////////////////////////////////////////////////
void CAsyncIOFileRequest::ComputeSortKey(uint64 nCurrentKeyInProgress)
{
	if (m_bSortKeyComputed)
		return;

	m_bSortKeyComputed = true;

	if (m_strFileName.empty())
	{
		m_eMediaType = eStreamSourceTypeMemory;
		m_nSortKey = m_ePriority;
		return;
	}
	if (m_bCanceled)
	{
		m_nSortKey = 0;
		return;
	}
	if (m_bNoSorting)
	{
		m_nSortKey = s_nSequenceCounter++;
		return;
	}

	const int g_nMaxPath = 0x800;
	char szFullPathBuf[g_nMaxPath];
	const char *szFullPath = gEnv->pCryPak->AdjustFileName(m_strFileName.c_str(), szFullPathBuf, ICryPak::FOPEN_HINT_QUIET);

	bool bFileInMemory = false;
	ICryPak* pCryPak = gEnv->pCryPak;
	ZipDir::Cache* pZip = 0;
	unsigned int archFlags;
	ZipDir::FileEntry* pFileEntry = ((CCryPak*)pCryPak)->FindPakFileEntry( szFullPath, archFlags, &pZip, false );
	if (pFileEntry)
	{
		m_nDiskOffset = pZip->GetPakFileOffsetOnMedia() + (uint64)pFileEntry->nFileDataOffset;
		bFileInMemory = pZip->IsInMemory();

		m_nSizeOnMedia = pFileEntry->desc.lSizeCompressed;
		m_nFileSize = pFileEntry->desc.lSizeUncompressed;

		m_pakFile = pZip->GetFilePath();
	} else {
		m_eMediaType = eStreamSourceTypeMemory;
		m_nSortKey = m_ePriority;
		return;
	}

	// group items by priority, then by snapped request time, then sort by disk offset
	m_nDiskOffset += m_nRequestedOffset;
	m_nTimeGroup = (uint64)(gEnv->pTimer->GetCurrTime() / max(1, g_cvars.sys_streaming_requests_grouping_time_period));
	uint64 nPrioriry = m_ePriority;

	int64 nDiskOffsetKB = m_nDiskOffset >> 10; // KB
	m_nSortKey = (nDiskOffsetKB) | (((uint64)m_nTimeGroup) << 30) | (nPrioriry << 60);

	// make sure we do not break incremental head movement within time group on every new request
	if(m_nSortKey <= nCurrentKeyInProgress)
	{
		m_nTimeGroup++;
		m_nSortKey = (nDiskOffsetKB) | (((uint64)m_nTimeGroup) << 30) | (nPrioriry << 60);
	}

//	m_nSortKey = nDiskOffsetKB;
	
	// Highest priority for files that are in memory already and can be loaded very fast.
	if (bFileInMemory)
	{
		m_eMediaType = eStreamSourceTypeMemory;
		m_nSortKey = m_ePriority;
	} else {
		// check if data is installed to HDD
		if (pCryPak->IsInstalledToHDD(pZip->GetFilePath()))
		{
			m_eMediaType = eStreamSourceTypeHDD;
		} else {
			m_eMediaType = eStreamSourceTypeDisc;
		}
	}

	// Update Streaming statistics.
	if (g_pStreamingStatistcs && m_nSizeOnMedia != 0)
	{
		m_bStatsUpdated = true;
		IStreamEngine::SStatistics &stats = *g_pStreamingStatistcs;
	
		// if the file is not compressed then it will only read the requested size
		uint32 nReadSize = m_nSizeOnMedia;
		if (m_nSizeOnMedia == m_nFileSize)
			nReadSize = m_nRequestedSize;

		CryInterlockedAdd( &stats.nPendingReadBytes,nReadSize );
		CryInterlockedAdd( &stats.typeInfo[m_eType].nPendingReadBytes,nReadSize );
	}
}
//////////////////////////////////////////////////////////////////////////
bool CAsyncIOFileRequest::IgnoreOutofTmpMem() const
{
	if (m_pReadStream &&
		(m_pReadStream->GetParams().ePriority == estpUrgent ||
		m_pReadStream->GetParams().nFlags & IStreamEngine::FLAGS_IGNORE_TMP_OUT_OF_MEM))
	{
		return true;
	}

	return false;
}
